#front matter
rm(list=ls())
library(foreign)
library(Hmisc)
library(timeSeries)
library(lmtest)


##### UNITED STATES ######
#clear Texas out
rm(list=ls())

###Load Partisanship Data###
#setwd("/Volumes/MONOGAN/DISSERTATION/partisanship/data/")
party<-read.csv('MacropartisanshipLeaners.csv',header=TRUE)
party$quarter<-NA
party$quarter[party$month<=3]<-1
party$quarter[party$month>3 & party$month<=6]<-2
party$quarter[party$month>6 & party$month<=9]<-3
party$quarter[party$month>9 & party$month<=12]<-4
party$id<-100*party$year+party$quarter
party$macropartisanship<-100*party$Democrat/(party$Democrat+party$Republican)
#party<-subset(party, select=-IntMethod)
party<-subset(party, select=c(SampleSize,Republican,Democrat,macropartisanship,id,year,quarter))

#collapse by quarter
party.2<-aggregate(party, by=list(party$id), FUN=mean)
party.2<-subset(party.2, select=-c(Group.1))
party.2$macropartisanship.q<-as.vector(sapply(split(party,party$id),function(x) weighted.mean(x$macropartisanship, x$SampleSize)))
party.2$republican.q<-as.vector(sapply(split(party,party$id),function(x) weighted.mean(x$Republican, x$SampleSize)))
#party.2<-subset(party.2, select=-id)
party.2<-subset(party.2, select=c(year,quarter,macropartisanship.q,republican.q))


###Load Consumer Sentiment Data###
consumer<-read.csv('consumerSentiment.csv')

###Load Presidential Approval Data###
approval<-read.csv('presApprovalGallup.csv')
approval$id<-100*approval$year+approval$quarter

###Load Control Variables###
controls<-read.csv('controlVars.csv')

#collapse by quarter
approval.2<-aggregate(approval, by=list(approval$id), FUN=mean)
approval.2<-subset(approval.2, select=-c(Group.1,month))
approval.2$approve.q<-as.vector(sapply(split(approval,approval$id),function(x) weighted.mean(x$approve, x$sample)))
approval.2<-subset(approval.2, select=-id)

###Merge Four Data Sets###
merge.1<-merge(x=party.2,y=consumer,by.x=c('year','quarter'),by.y=c('Year','Quarter'),all.y=TRUE)
merge.2<-merge(x=merge.1,y=controls,by=c('year','quarter'),all=TRUE)
full<-merge(x=approval.2,y=merge.2,by=c('year','quarter'),all.y=TRUE)

##What's the time frame? 1984-2010, Need predictors since 1984##
us.short<-subset(full,year>=1984 & year<=2010, select=c(year,quarter,approve.q,macropartisanship.q,republican.q,Index,party,president,LAtimes,hispFB))[-c(1:2),]#Drop first two observations of 1984, which do not include leaners.

##Create TS union with lags##
#Party and president missing a couple of values for coding reasons. LVCF correctly codes in this case.
party.0<-interpNA(us.short$party,method='before')
president.0<-interpNA(us.short$president,method='before')
#Approval is missing in the 3rd quarter of 1972 & 1976.
approve.0<-interpNA(us.short$approve.q,method='before')
approve.q<-ts(approve.0*party.0)
macropartisanship.q.0<-interpNA(us.short$macropartisanship.q,method='before')
macropartisanship.q<-ts(macropartisanship.q.0)
republican.q.0<-interpNA(us.short$republican.q,method='before')
republican.q<-ts(republican.q.0)
party<-ts(party.0)
president<-ts(president.0)
sentiment<-ts(us.short$Index*party.0)
year<-ts(us.short$year)
quarter<-ts(us.short$quarter)
pol.mod<-lm(approve.q~sentiment);summary(pol.mod)
political<-ts(pol.mod$residuals)
l.approve<-lag(approve.q,-1)
l.macropartisanship<-lag(macropartisanship.q,-1)
l.republican<-lag(republican.q,-1)
l.sentiment<-lag(sentiment,-1)
l.political<-lag(political,-1)
id<-ts(us.short$year*100+us.short$quarter)
LAtimes<-ts(us.short$LAtimes)
hispFB<-ts(us.short$hispFB)
us.time.0<-as.data.frame(ts.union(political,approve.q,macropartisanship.q,republican.q,sentiment,year,quarter,l.political,l.approve,l.macropartisanship,l.republican,l.sentiment,party,president,LAtimes,hispFB,id))
us.time<-subset(us.time.0,year>=1984)


##### CALIFORNIA #####
ca.0<-read.spss("umembargo_cumulative.sav", use.value.labels=FALSE, to.data.frame=TRUE)
#codebook<-read.spss("umembargo_cumulative.sav", use.value.labels=TRUE, to.data.frame=TRUE)

#Identify which quarter
ca.quarters<-read.csv('fieldQuarters.csv')
ca<-merge(x=ca.quarters, y=ca.0, by='POLLNUM', all.y=TRUE)
ca$id<-100*ca$YEAR+ca$quarter

#Recode Republicanism into a dummy
ca$PID7[ca$PID7>7]<-NA
ca$dem<-as.numeric(ca$PID7>4)*100
ca$rep<-as.numeric(ca$PID7<4)*100

#Create complete, Hispanic, and white subsets
california<-subset(ca,select=c(dem,rep,YEAR,quarter,id))
hispanic<-subset(ca,RACE2==3,select=c(dem,rep,YEAR,quarter,id))
white<-subset(ca,RACE2==1,select=c(dem,rep,YEAR,quarter,id))

#Build aggregate california datasets
ca.total.q<-aggregate(california, by=list(california$id), FUN=mean, na.rm=TRUE)
ca.hispanic.q<-aggregate(hispanic, by=list(hispanic$id), FUN=mean, na.rm=TRUE)
ca.white.q<-aggregate(white, by=list(white$id), FUN=mean, na.rm=TRUE)

#cleaning
ca.total.q<-subset(ca.total.q, subset=!is.nan(ca.total.q$dem)&YEAR>=1984,select=-c(Group.1,id))#subset to be the same length as the demographic subsets
ca.hispanic.q<-subset(ca.hispanic.q, subset=!is.nan(ca.hispanic.q$dem)&YEAR>=1984,select=-c(Group.1,id))
ca.white.q<-subset(ca.white.q, subset=!is.nan(ca.white.q$dem)&YEAR>=1984,select=-c(Group.1,id))

#append new data (file name: caUpdates3.xlsx)
ca.updates<-read.csv('caPartyUpdatesLeaners.csv',header=TRUE)
ca.updates$repAll<-ca.updates$repAll+ca.updates$indAll.R
ca.updates$demAll<-ca.updates$demAll+ca.updates$indAll.D
ca.updates$repWhite<-ca.updates$repWhite+ca.updates$indWhite.R
ca.updates$demWhite<-ca.updates$demWhite+ca.updates$indWhite.D
ca.updates$repHisp<-ca.updates$repHisp+ca.updates$indHisp.R
ca.updates$demHisp<-ca.updates$demHisp+ca.updates$indHisp.D

total.up<-subset(ca.updates,select=c(sampAll,repAll,demAll,YEAR,quarter))
total.up$dem<-100*total.up$demAll/total.up$sampAll
total.up$rep<-100*total.up$repAll/total.up$sampAll
total.up<-total.up[,c('dem','rep','YEAR','quarter')]
ca.total.q<-rbind(ca.total.q,total.up)

hispanic.up<-subset(ca.updates,select=c(sampHisp,repHisp,demHisp,YEAR,quarter))
hispanic.up$dem<-100*hispanic.up$demHisp/hispanic.up$sampHisp
hispanic.up$rep<-100*hispanic.up$repHisp/hispanic.up$sampHisp
hispanic.up<-hispanic.up[,c('dem','rep','YEAR','quarter')]
ca.hispanic.q<-rbind(ca.hispanic.q,hispanic.up)

white.up<-subset(ca.updates,select=c(sampWhite,repWhite,demWhite,YEAR,quarter))
white.up$dem<-100*white.up$demWhite/white.up$sampWhite
white.up$rep<-100*white.up$repWhite/white.up$sampWhite
white.up<-white.up[,c('dem','rep','YEAR','quarter')]
ca.white.q<-rbind(ca.white.q,white.up)

#merge with national data
full.ca<-merge(y=ca.total.q,x=us.short,by.y=c('YEAR','quarter'),by.x=c('year','quarter'),all.x=TRUE)
full.ca.hisp<-merge(y=ca.hispanic.q,x=us.short,by.y=c('YEAR','quarter'),by.x=c('year','quarter'),all.x=TRUE)
full.ca.white<-merge(y=ca.white.q,x=us.short,by.y=c('YEAR','quarter'),by.x=c('year','quarter'),all.x=TRUE)

##Create TS union with lags##
#Party and president missing a couple of values for coding reasons. LVCF correctly codes in this case.
party.0<-interpNA(full.ca$party,method='before')
president.0<-interpNA(full.ca$president,method='before')
#Approval is missing in the 3rd quarter of 1972 & 1976.
approve.0<-interpNA(full.ca$approve.q,method='before')
approve.q<-ts(approve.0*party.0)
dem.0<-interpNA(full.ca$dem,method='before')
rep.0<-interpNA(full.ca$rep,method='before')
caPartisanship<-100*dem.0/(dem.0+rep.0)
caPartisanship.q<-ts(caPartisanship)

#Remaining Series
party<-ts(party.0)
president<-ts(president.0)
sentiment<-ts(full.ca$Index*party.0)
year<-ts(full.ca$year)
quarter<-ts(full.ca$quarter)
pol.mod<-lm(approve.q~sentiment);summary(pol.mod)
political<-ts(pol.mod$residuals)
republican<-ts(rep.0)
l.republican<-lag(republican,-1)
l.approve<-lag(approve.q,-1)
l.caPartisanship<-lag(caPartisanship.q,-1)
l.sentiment<-lag(sentiment,-1)
l.political<-lag(political,-1)
id<-ts(full.ca$year*100+full.ca$quarter)
ca.time<-as.data.frame(ts.union(political,approve.q,caPartisanship.q,sentiment,year,quarter,l.political,l.approve,l.caPartisanship,l.sentiment,party,president,LAtimes,hispFB,id,republican,l.republican)[-107,])
#ca.test<-as.data.frame(ca.time)

#Repeat for Hispanic data
party.hisp.0<-interpNA(full.ca.hisp$party,method='before')
president.hisp.0<-interpNA(full.ca.hisp$president,method='before')
approve.hisp.0<-interpNA(full.ca.hisp$approve.q,method='before')
approve.hisp.q<-ts(approve.hisp.0*party.0)
dem.hisp.0<-interpNA(full.ca.hisp$dem,method='before')
rep.hisp.0<-interpNA(full.ca.hisp$rep,method='before')
caPartisanship.hisp<-100*dem.hisp.0/(dem.hisp.0+rep.hisp.0)
caPartisanship.hisp.q<-ts(caPartisanship.hisp)
party.hisp<-ts(party.hisp.0)
president.hisp<-ts(president.hisp.0)
sentiment.hisp<-ts(full.ca.hisp$Index*party.hisp.0)
year.hisp<-ts(full.ca.hisp$year)
quarter.hisp<-ts(full.ca.hisp$quarter)
pol.mod.hisp<-lm(approve.hisp.q~sentiment.hisp);summary(pol.mod.hisp)
political.hisp<-ts(pol.mod.hisp$residuals)
republican.hisp<-ts(rep.hisp.0)
l.republican.hisp<-lag(republican.hisp,-1)
l.approve.hisp<-lag(approve.hisp.q,-1)
l.caPartisanship.hisp<-lag(caPartisanship.hisp.q,-1)
l.sentiment.hisp<-lag(sentiment.hisp,-1)
l.political.hisp<-lag(political.hisp,-1)
id.hisp<-ts(full.ca.hisp$year*100+full.ca.hisp$quarter)
ca.time.hisp<-as.data.frame(ts.union(political.hisp,approve.hisp.q,caPartisanship.hisp.q,sentiment.hisp,year.hisp,quarter.hisp,l.political.hisp,l.approve.hisp,l.caPartisanship.hisp,l.sentiment.hisp,party.hisp,president.hisp,LAtimes,hispFB,id.hisp,republican.hisp,l.republican.hisp)[-107,])

#Repeat for white data
party.white.0<-interpNA(full.ca.white$party,method='before')
president.white.0<-interpNA(full.ca.white$president,method='before')
approve.white.0<-interpNA(full.ca.white$approve.q,method='before')
approve.white.q<-ts(approve.white.0*party.0)
dem.white.0<-interpNA(full.ca.white$dem,method='before')
rep.white.0<-interpNA(full.ca.white$rep,method='before')
caPartisanship.white<-100*dem.white.0/(dem.white.0+rep.white.0)
caPartisanship.white.q<-ts(caPartisanship.white)
party.white<-ts(party.white.0)
president.white<-ts(president.white.0)
sentiment.white<-ts(full.ca.white$Index*party.white.0)
year.white<-ts(full.ca.white$year)
quarter.white<-ts(full.ca.white$quarter)
pol.mod.white<-lm(approve.white.q~sentiment.white);summary(pol.mod.white)
political.white<-ts(pol.mod.white$residuals)
republican.white<-ts(rep.white.0)
l.republican.white<-lag(republican.white,-1)
l.approve.white<-lag(approve.white.q,-1)
l.caPartisanship.white<-lag(caPartisanship.white.q,-1)
l.sentiment.white<-lag(sentiment.white,-1)
l.political.white<-lag(political.white,-1)
id.white<-ts(full.ca.white$year*100+full.ca.white$quarter)
ca.time.white<-as.data.frame(ts.union(political.white,approve.white.q,caPartisanship.white.q,sentiment.white,year.white,quarter.white,l.political.white,l.approve.white,l.caPartisanship.white,l.sentiment.white,party.white,president.white,LAtimes,hispFB,id.white,republican.white,l.republican.white)[-107,])
table(ca.time.white$year)

###GOP ONLY MODELS###
ca.intervention.hisp<-lm(republican.hisp~party.hisp+political.hisp+sentiment.hisp+l.republican.hisp+as.factor(president.hisp)+as.numeric(id.hisp>=199503),data=ca.time.hisp,subset=year>=1984); summary(ca.intervention.hisp)
ca.intervention.white<-lm(republican.white~party.white+political.white+sentiment.white+l.republican.white+as.factor(president.white)+as.numeric(id.white>=199503),data=ca.time.white,subset=year>=1984); summary(ca.intervention.white)
ca.intervention<-lm(republican~party+political+sentiment+l.republican+as.factor(president)+as.numeric(id>=199503),data=ca.time,subset=year>=1984); summary(ca.intervention)
us.intervention<-lm(republican.q~party+political+sentiment+l.republican+as.factor(president)+as.numeric(id>=199503),data=us.time[-106,],subset=year>=1984); summary(us.intervention)

###MACROPARTISANSHIP WITH LEANER MODELS###
leaner.ca.intervention.hisp<-lm(caPartisanship.hisp.q~party.hisp+political.hisp+sentiment.hisp+l.caPartisanship.hisp+as.factor(president.hisp)+as.numeric(id.hisp>=199503),data=ca.time.hisp,subset=year>=1984); summary(leaner.ca.intervention.hisp)
leaner.ca.intervention.white<-lm(caPartisanship.white.q~party.white+political.white+sentiment.white+l.caPartisanship.white+as.factor(president.white)+as.numeric(id.white>=199503),data=ca.time.white,subset=year>=1984); summary(leaner.ca.intervention.white)
leaner.ca.intervention<-lm(caPartisanship.q~party+political+sentiment+l.caPartisanship+as.factor(president)+as.numeric(id>=199503),data=ca.time,subset=year>=1984); summary(leaner.ca.intervention)
leaner.us.intervention<-lm(macropartisanship.q~party+political+sentiment+l.macropartisanship+as.factor(president)+as.numeric(id>=199503),data=us.time[-106,],subset=year>=1984); summary(leaner.us.intervention)

#Breusch-Godfrey tests
bgtest(us.intervention)
bgtest(ca.intervention)
bgtest(ca.intervention.hisp)
bgtest(ca.intervention.white)
bgtest(leaner.us.intervention)
bgtest(leaner.ca.intervention)
bgtest(leaner.ca.intervention.hisp)
bgtest(leaner.ca.intervention.white)
